# -*- coding: utf-8 -*-
from urllib import parse

import scrapy
from scrapy import signals  # 信号量
# from scrapy.xlib.pydispatch import dispatcher  # 分发器

from ArticleSpider.items import JobboleArticleItem, ArticleItemLoader
from ArticleSpider.utils.common import get_md5


class JobboleSpider(scrapy.Spider):
    name = 'jobbole'
    allowed_domains = ['blog.jobbole.com']
    start_urls = ['http://blog.jobbole.com/all-posts/']
    # start_urls = ['http://blog.jobbole.com/boby/']

    # def __init__(self):
    #     self.browser = webdriver.Chrome()
    #     super(JobboleSpider, self).__init__()
    #     dispatcher.connect(self.spider_closed,signals.spider_closed)
    #
    # def spider_closed(self):
    #     #当爬虫退出的时候关闭Chrome
    #     print('spider closed')
    #     self.browser.quit()
    # 收集伯乐在线所有404页面的url以及404页面数
    handle_httpstatus_list = [404]

    def __init__(self):
        self.fail_urls = []
        # dispatcher.connect(self.handle_failed_url, signals.spider_closed)

    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        spider = super(JobboleSpider, cls).from_crawler(crawler, *args, **kwargs)
        spider.fail_urls = []
        crawler.signals.connect(spider.handle_failed_url, signals.spider_closed)
        return spider

    def handle_failed_url(self, spider, reason):
        self.crawler.stats.set_value('failed_urls', ','.join(self.fail_urls))

    def parse(self, response):
        """
        1.获取文章列表页的文章url交给scrapy下载后并进行解析
        2.获取下一页的url交给scrapy进行下载，下载完成后交给parse解析
        """
        """
        解析文章列表页中的所有文章url交给scrapy下载并进行解析
        """
        if response.status == 404:
            self.fail_urls.append(response.url)
            self.crawler.stats.inc_value("failed_urls")

        post_nodes = response.css('#archive .post-thumb a')
        for post_node in post_nodes:
            img_url = post_node.css('img::attr(src)').extract_first()
            # img_url = [img_url if 'http:' in img_url else ('http:' + img_url)]
            post_url = post_node.css('::attr(href)').extract_first()
            yield scrapy.Request(url=parse.urljoin(response.url, post_url), meta={'img_url': img_url},
                                 callback=self.parse_detail)
        next_url = response.css('.next.page-numbers::attr(href)').extract_first()
        # 获取下一页的url交给scrapy下载并进行解析
        if next_url:
            yield scrapy.Request(url=next_url, callback=self.parse)

    def parse_detail(self, response):
        # items = JobboleArticleItem()
        # title = response.xpath('//div[@class="entry-header"]/h1/text()')[0].extract()
        # create_date = response.xpath('//p[@class="entry-meta-hide-on-mobile"]/text()').extract()[0].strip().replace('·','').strip()
        # praise_nums = int(response.xpath("//span[contains(@class,'vote-post-up')]/h10/text()").extract_first())
        # fav_nums = response.xpath("//span[contains(@class,'bookmark-btn')]/text()").extract_first()
        # try:
        #     if re.match('.*?(\d+).*', fav_nums).group(1):
        #         fav_nums = int(re.match('.*?(\d+).*', fav_nums).group(1))
        #     else:
        #         fav_nums = 0
        # except:
        #     fav_nums = 0
        # comment_nums = response.xpath('//a[contains(@href,"#article-comment")]/span/text()').extract()[0]
        # try:
        #     if re.match('.*?(\d+).*',comment_nums).group(1):
        #         comment_nums = int(re.match('.*?(\d+).*',comment_nums).group(1))
        #     else:
        #         comment_nums = 0
        # except:
        #     comment_nums = 0
        # contente = response.xpath('//div[@class="entry"]').extract()[0]
        # tag_list = response.xpath('//p[@class="entry-meta-hide-on-mobile"]/a/text()').extract()
        # tag_list = [tag for tag in tag_list if not tag.strip().endswith('评论')]
        # tags = ",".join(tag_list)
        # items['title'] = title
        # try:
        #     create_date = datetime.datetime.strptime(create_date,'%Y/%m/%d').date()
        # except:
        #     create_date = datetime.datetime.now()
        # items['date'] = create_date
        # items['url'] = response.url
        # items['url_object_id'] = get_md5(response.url)
        # items['img_url'] = [img_url]
        # items['praise_nums'] = praise_nums
        # items['fav_nums'] = fav_nums
        # items['comment_nums'] = comment_nums
        # items['content'] = contente
        # items['tags'] = tags

        # 通过item loader加载item

        front_image_url = response.meta.get("front_image_url", "")  # 文章封面图
        item_loader = ArticleItemLoader(item=JobboleArticleItem(), response=response)
        item_loader.add_css("title", ".entry-header h1::text")
        item_loader.add_value("url", response.url)
        item_loader.add_value("url_object_id", get_md5(response.url))
        item_loader.add_css("create_date", "p.entry-meta-hide-on-mobile::text")
        item_loader.add_value("front_image_url", [front_image_url])
        item_loader.add_css("praise_nums", ".vote-post-up h10::text")
        item_loader.add_css("comment_nums", "a[href='#article-comment'] span::text")
        item_loader.add_css("fav_nums", ".bookmark-btn::text")
        item_loader.add_css("tags", "p.entry-meta-hide-on-mobile a::text")
        item_loader.add_css("content", "div.entry")

        article_item = item_loader.load_item()

        yield article_item

        # title = response.css('.entry-header h1::text')[0].extract()
        # create_date = response.css('p.entry-meta-hide-on-mobile::text').extract()[0].strip().replace('·','').strip()
        # praise_nums = int(response.css(".vote-post-up h10::text").extract_first()
        # fav_nums = response.css(".bookmark-btn::text").extract_first()
        # if re.match('.*?(\d+).*', fav_nums).group(1):
        #     fav_nums = int(re.match('.*?(\d+).*', fav_nums).group(1))
        # else:
        #     fav_nums = 0
        # comment_nums = response.css('a[href="#article-comment"] span::text').extract()[0]
        # if re.match('.*?(\d+).*', comment_nums).group(1):
        #     comment_nums = int(re.match('.*?(\d+).*', comment_nums).group(1))
        # else:
        #     comment_nums = 0
        # content = response.css('.entry').extract()[0]
        # tag_list = response.css('p.entry-meta-hide-on-mobile a::text')
        # tag_list = [tag for tag in tag_list if not tag.strip().endswith('评论')]
        # tags = ",".join(tag_list)
        # xpath选择器 /@href    /text()

